{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": "tensor([[ 0.,  1.,  2.,  3.,  4.,  5.,  6.,  7.],\n        [ 8.,  9., 10., 11., 12., 13., 14., 15.]])\ntensor([[ 0.,  2.,  4.,  0.,  0.,  0., 12., 14.],\n        [16.,  0.,  0., 22.,  0., 26.,  0.,  0.]])\ntensor([[0., 0., 0., 0., 0., 0., 0., 0.],\n        [0., 0., 0., 0., 0., 0., 0., 0.]])\n"
    }
   ],
   "source": [
    "%matplotlib inline\n",
    "import torch\n",
    "import torch.nn as nn \n",
    "import numpy as np \n",
    "import d2lzh as d2l\n",
    "\n",
    "# 随机的将隐藏层某些单元的输出值设置为０\n",
    "def dropout(X,drop_prob):\n",
    "    X = X.float()\n",
    "    assert 0 <= drop_prob <= 1\n",
    "    keep_prob = 1 - drop_prob\n",
    "    if keep_prob == 0:\n",
    "        return torch.zeros_like(X)\n",
    "\n",
    "    mask = (torch.randn(X.shape) < keep_prob).float()\n",
    "\n",
    "    return mask * X / keep_prob\n",
    "\n",
    "X = torch.arange(16).view(2,8)\n",
    "print(dropout(X,0))\n",
    "print(dropout(X,0.5))\n",
    "print(dropout(X,1.0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": "changeing train\nepoch 1,loss 0.0049,train acc 0.518,test_acc 0.725\nepoch 2,loss 0.0023,train acc 0.778,test_acc 0.811\nepoch 3,loss 0.0020,train acc 0.816,test_acc 0.812\nepoch 4,loss 0.0018,train acc 0.833,test_acc 0.835\nepoch 5,loss 0.0017,train acc 0.844,test_acc 0.843\n"
    }
   ],
   "source": [
    "num_inputs,num_outputs,num_hiddens1,num_hiddens2 = 784,10,256,256\n",
    "\n",
    "W1 = torch.tensor(np.random.normal(0,0.01,size=(num_inputs,num_hiddens1)),dtype=torch.float,requires_grad=True)\n",
    "b1 = torch.zeros(num_hiddens1,requires_grad=True)\n",
    "W2 = torch.tensor(np.random.normal(0,0.01,size=(num_hiddens1,num_hiddens2)),dtype=torch.float,requires_grad=True)\n",
    "b2 = torch.zeros(num_hiddens2,requires_grad=True)\n",
    "W3 = torch.tensor(np.random.normal(0,0.01,size=(num_hiddens2,num_outputs)),dtype=torch.float,requires_grad=True)\n",
    "b3 = torch.zeros(num_outputs,requires_grad=True)\n",
    "\n",
    "params = [W1,b1,W2,b2,W3,b3]\n",
    "\n",
    "drop_prob1,drop_prob2 = 0.2,0.5\n",
    "\n",
    "def net(X,is_training=True):\n",
    "    X = X.view(-1,num_inputs)\n",
    "    H1 = (torch.matmul(X,W1) + b1).relu()\n",
    "    if is_training:\n",
    "        H1 = dropout(H1,drop_prob1)  # 第一个隐藏层后接dropout\n",
    "    H2 = (torch.matmul(H1,W2) + b2).relu()\n",
    "    if is_training:\n",
    "        H2 = dropout(H2,drop_prob2)\n",
    "\n",
    "    return torch.matmul(H2,W3) + b3\n",
    "\n",
    "num_epochs,lr,batch_size = 5,100.0,256\n",
    "loss = torch.nn.CrossEntropyLoss()\n",
    "train_iter,test_iter = d2l.load_data_fashion_mnist(batch_size)\n",
    "d2l.train_ch3(net,train_iter,test_iter,loss,num_epochs,batch_size,params,lr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": "changeing train\nepoch 1,loss 0.0044,train acc 0.571,test_acc 0.727\nepoch 2,loss 0.0023,train acc 0.786,test_acc 0.823\nepoch 3,loss 0.0019,train acc 0.825,test_acc 0.823\nepoch 4,loss 0.0017,train acc 0.838,test_acc 0.843\nepoch 5,loss 0.0016,train acc 0.850,test_acc 0.848\n"
    }
   ],
   "source": [
    "net = nn.Sequential(\n",
    "    d2l.FlattenLayer(),\n",
    "    nn.Linear(num_inputs,num_hiddens1),\n",
    "    nn.ReLU(),\n",
    "    nn.Dropout(drop_prob1),\n",
    "    nn.Linear(num_hiddens1,num_hiddens2),\n",
    "    nn.ReLU(),\n",
    "    nn.Dropout(drop_prob2),\n",
    "    nn.Linear(num_hiddens2,10)\n",
    ")\n",
    "\n",
    "for param in net.parameters():\n",
    "    nn.init.normal_(param,mean=0,std=0.01)\n",
    "\n",
    "optimizer = torch.optim.SGD(net.parameters(),lr=0.5)\n",
    "d2l.train_ch3(net,train_iter,test_iter,loss,num_epochs,batch_size,None,None,optimizer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7-final"
  },
  "orig_nbformat": 2,
  "kernelspec": {
   "name": "python37764bitpytorchnotebookconda6e7a8693df0d4d92aca09d521275d23a",
   "display_name": "Python 3.7.7 64-bit ('pytorch_notebook': conda)"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}